﻿

using HtmlAgilityPack;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;

string searchText = "https://www.baidu.com/baidu?wd=1";
string searchText2 = "https://www.baidu.com";
await Search_baiduAsync(searchText);

async Task Search_baiduAsync(string url)         
{
    var socketsHttpHandler = new SocketsHttpHandler()
    {
        AllowAutoRedirect = true,// 默认为true,是否允许冲顶定向
        MaxAutomaticRedirections = 50,//最多重定向几次,默认50次                                          
        UseCookies = false// 是否自动处理cookie
    };
    HttpClient httpClient = new HttpClient(socketsHttpHandler);
    httpClient.DefaultRequestHeaders.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.50");
    httpClient.DefaultRequestHeaders.Add("Cookie", "BIDUPSID=BC88456742AA1FF88393F1DC523FDA3B; PSTM=1621751763; BD_UPN=12314753; __yjs_duid=1_f8046a51e2af9aef8f45769dd03752661621754963739; BDUSS=H5DQWpTNUh6T1dKc25XTm85Mng4OVZJOVplczRIS2NGTGt0OVhzT3J4TWlpbHRpRVFBQUFBJCQAAAAAAAAAAAEAAABDZ7gvvvjWrtDHs70AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACL9M2Ii~TNib; BDUSS_BFESS=H5DQWpTNUh6T1dKc25XTm85Mng4OVZJOVplczRIS2NGTGt0OVhzT3J4TWlpbHRpRVFBQUFBJCQAAAAAAAAAAAEAAABDZ7gvvvjWrtDHs70AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACL9M2Ii~TNib; BAIDUID=538EB63CAFCD589796F445AD9451262B:SL=0:NR=10:FG=1; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; BA_HECTOR=0h85a025048kak2gr91h6d3kq0q; BDRCVFR[OEHfjv-pq1f]=K7PavkeTFPTUAN8ULuEQhPEUi4WU6; delPer=0; BD_CK_SAM=1; PSINO=3; H_PS_PSSID=; sug=3; sugstore=0; ORIGIN=2; bdime=0; H_PS_645EC=9fe7VqXx9piEiCi5qhsoQ5pnAxn7+YD3Lazb1hNLSZvtL5Q9BYKyZM/VuxF0xW99ifY; BAIDUID_BFESS=538EB63CAFCD589796F445AD9451262B:SL=0:NR=10:FG=1; BDSVRTM=0");

    HttpResponseMessage response = await httpClient.GetAsync(url); 
    response.EnsureSuccessStatusCode();
    var str = await response.Content.ReadAsStringAsync();
    Download(str);
    StreamReader sr = File.OpenText("123.html");
    HtmlDocument htmlDocument = new HtmlDocument();
    htmlDocument.Load(sr);
    HtmlNode version = htmlDocument.DocumentNode.SelectSingleNode("html/body");
    string mask = "[\n\t ]";
    Regex reg = new Regex(mask);
    string res = reg.Replace(version.InnerText, "");
    Console.Write(res);
}
string Download(string html)
{//下载指定网页的内容
    try
    {
        string fileName = "123.html";
        File.WriteAllText(fileName, html, Encoding.UTF8);
        return html;
    }
    catch (Exception ex)
    {
        Console.WriteLine(ex.Message);
        return "";
    }
}